{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>content_id</th>\n",
       "      <th>content</th>\n",
       "      <th>subject</th>\n",
       "      <th>sentiment_value</th>\n",
       "      <th>sentiment_word</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>vUXizsqexyZVRdFH</td>\n",
       "      <td>因为森林人即将换代，这套系统没必要装在一款即将换代的车型上，因为肯定会影响价格。</td>\n",
       "      <td>价格</td>\n",
       "      <td>0</td>\n",
       "      <td>影响</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4QroPd9hNfnCHVt7</td>\n",
       "      <td>四驱价格貌似挺高的，高的可以看齐XC60了，看实车前脸有点违和感。不过大众的车应该不会差。</td>\n",
       "      <td>价格</td>\n",
       "      <td>-1</td>\n",
       "      <td>高</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>QmqJ2AvM5GplaRyz</td>\n",
       "      <td>斯柯达要说质量，似乎比大众要好一点，价格也低一些，用料完全一样。我听说过野帝，但没听说过你说...</td>\n",
       "      <td>价格</td>\n",
       "      <td>1</td>\n",
       "      <td>低</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>KMT1gFJiU4NWrVDn</td>\n",
       "      <td>这玩意都是给有钱任性又不懂车的土豪用的，这价格换一次我妹夫EP020可以换三锅了</td>\n",
       "      <td>价格</td>\n",
       "      <td>-1</td>\n",
       "      <td>有钱任性</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>nVIlGd5yMmc37t1o</td>\n",
       "      <td>17价格忒高，估计也就是14-15左右。</td>\n",
       "      <td>价格</td>\n",
       "      <td>-1</td>\n",
       "      <td>高</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         content_id                                            content  \\\n",
       "0  vUXizsqexyZVRdFH           因为森林人即将换代，这套系统没必要装在一款即将换代的车型上，因为肯定会影响价格。   \n",
       "1  4QroPd9hNfnCHVt7      四驱价格貌似挺高的，高的可以看齐XC60了，看实车前脸有点违和感。不过大众的车应该不会差。   \n",
       "2  QmqJ2AvM5GplaRyz  斯柯达要说质量，似乎比大众要好一点，价格也低一些，用料完全一样。我听说过野帝，但没听说过你说...   \n",
       "3  KMT1gFJiU4NWrVDn           这玩意都是给有钱任性又不懂车的土豪用的，这价格换一次我妹夫EP020可以换三锅了   \n",
       "4  nVIlGd5yMmc37t1o                            17价格忒高，估计也就是14-15左右。      \n",
       "\n",
       "  subject  sentiment_value sentiment_word  \n",
       "0      价格                0             影响  \n",
       "1      价格               -1              高  \n",
       "2      价格                1              低  \n",
       "3      价格               -1           有钱任性  \n",
       "4      价格               -1              高  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = \"data/train.csv\"\n",
    "df = pd.read_csv(path)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(9947, 5)\n",
      "(8290,)\n"
     ]
    }
   ],
   "source": [
    "print(df.shape)\n",
    "print(df[\"content_id\"].unique().shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>-1</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>all</th>\n",
       "      <th>+_mean</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>subject</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>价格</th>\n",
       "      <td>145</td>\n",
       "      <td>1014</td>\n",
       "      <td>114</td>\n",
       "      <td>1273</td>\n",
       "      <td>129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>内饰</th>\n",
       "      <td>150</td>\n",
       "      <td>271</td>\n",
       "      <td>115</td>\n",
       "      <td>536</td>\n",
       "      <td>132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>动力</th>\n",
       "      <td>378</td>\n",
       "      <td>1970</td>\n",
       "      <td>384</td>\n",
       "      <td>2732</td>\n",
       "      <td>381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>外观</th>\n",
       "      <td>111</td>\n",
       "      <td>263</td>\n",
       "      <td>115</td>\n",
       "      <td>489</td>\n",
       "      <td>113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>安全性</th>\n",
       "      <td>93</td>\n",
       "      <td>380</td>\n",
       "      <td>100</td>\n",
       "      <td>573</td>\n",
       "      <td>96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>操控</th>\n",
       "      <td>124</td>\n",
       "      <td>606</td>\n",
       "      <td>306</td>\n",
       "      <td>1036</td>\n",
       "      <td>215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>油耗</th>\n",
       "      <td>138</td>\n",
       "      <td>793</td>\n",
       "      <td>151</td>\n",
       "      <td>1082</td>\n",
       "      <td>144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>空间</th>\n",
       "      <td>67</td>\n",
       "      <td>221</td>\n",
       "      <td>154</td>\n",
       "      <td>442</td>\n",
       "      <td>110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>舒适性</th>\n",
       "      <td>256</td>\n",
       "      <td>564</td>\n",
       "      <td>111</td>\n",
       "      <td>931</td>\n",
       "      <td>183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>配置</th>\n",
       "      <td>154</td>\n",
       "      <td>579</td>\n",
       "      <td>120</td>\n",
       "      <td>853</td>\n",
       "      <td>137</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          -1     0    1   all  +_mean\n",
       "subject                              \n",
       "价格       145  1014  114  1273     129\n",
       "内饰       150   271  115   536     132\n",
       "动力       378  1970  384  2732     381\n",
       "外观       111   263  115   489     113\n",
       "安全性       93   380  100   573      96\n",
       "操控       124   606  306  1036     215\n",
       "油耗       138   793  151  1082     144\n",
       "空间        67   221  154   442     110\n",
       "舒适性      256   564  111   931     183\n",
       "配置       154   579  120   853     137"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnt_data = []\n",
    "for k, gp in df.groupby(\"subject\"):\n",
    "    cnt = gp.groupby(\"sentiment_value\").count()[\"content\"].to_dict()\n",
    "    cnt[\"subject\"] = k\n",
    "    cnt_data.append(cnt)\n",
    "dfcnt = pd.DataFrame(cnt_data)[[\"subject\", -1, 0, 1]]\n",
    "dfcnt[\"all\"] = dfcnt[-1] + dfcnt[1] + dfcnt[0]\n",
    "dfcnt[\"+_mean\"] = ((dfcnt[-1] + dfcnt[1]) / 2).map(int)\n",
    "dfcnt.index = dfcnt[\"subject\"]\n",
    "del dfcnt[\"subject\"]\n",
    "dfcnt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 对subject处理\n",
    "tmp = []\n",
    "for id_, gp in df.groupby(\"content_id\"):\n",
    "    content= gp[\"content\"].values[0]\n",
    "    subjects = gp[\"subject\"].tolist()\n",
    "    tmp.append((id_, content, subjects))\n",
    "    \n",
    "random.shuffle(tmp) # 打乱顺序\n",
    "content_id, content, subjects = zip(*tmp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.preprocessing import MultiLabelBinarizer\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.externals import joblib\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_selection import SelectKBest, SelectPercentile\n",
    "from sklearn.feature_selection import chi2\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.svm import LinearSVC\n",
    "\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn import svm\n",
    "from sklearn.model_selection import KFold, cross_val_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 多标签的处理\n",
    "subject_mlb = MultiLabelBinarizer()\n",
    "y_sub = subject_mlb.fit_transform(subjects)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/python3/lib/python3.6/site-packages/sklearn/feature_extraction/text.py:1089: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  if hasattr(X, 'dtype') and np.issubdtype(X.dtype, np.float):\n"
     ]
    }
   ],
   "source": [
    "# 文本处理\n",
    "vec =TfidfVectorizer(token_pattern=r\"(?u)\\b\\S+\\b\",\n",
    "                                ngram_range=(1, 2), \n",
    "                                 analyzer=\"char\",\n",
    "                                 max_df=0.6, \n",
    "                                 min_df=15)\n",
    "\n",
    "X = vec.fit_transform(content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.76708861 0.76371308 0.75844595 0.74915541 0.75929054 0.74155405\n",
      " 0.76097973]\n"
     ]
    }
   ],
   "source": [
    "clf =  OneVsRestClassifier(estimator=RandomForestClassifier(n_estimators=200,\n",
    "                                                            random_state =50,\n",
    "                                                            n_jobs=-1,\n",
    "                                                            max_features=\"auto\",\n",
    "                                                           ))\n",
    "\n",
    "k_fold = KFold(n_splits=7)\n",
    "scores = cross_val_score(clf, X, y_sub, cv=k_fold, n_jobs=-1)\n",
    "print(scores)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " @@@ngram:1,3\n",
    "[0.72902604 0.75024108 0.73069498 0.74131274 0.76351351 0.76544402\n",
    " 0.73841699 0.72104247]\n",
    "<br>\n",
    " @@@ngram:1,2\n",
    "[0.72999036 0.76277724 0.74034749 0.75096525 0.76254826 0.75675676\n",
    " 0.73552124 0.72393822]\n",
    " <br>\n",
    " #######<br>\n",
    " @ngram:1,2 @min_df:10\n",
    "<br>[0.73866924 0.76856316 0.74710425 0.75579151 0.76351351 0.76640927\n",
    " 0.75       0.73648649]\n",
    " <br>\n",
    " #######<br>\n",
    " @ngram:1,2 @min_df:15<br>\n",
    " [0.7415622  0.77242044 0.74131274 0.76447876 0.76833977 0.77027027\n",
    " 0.75579151 0.73359073]\n",
    " <br>\n",
    " #######<br>\n",
    " @ngram:1,2 @min_df:15@max_df:0.6<br>\n",
    "[0.76181292 0.75891996 0.76544402 0.74131274 0.75096525 0.74420849\n",
    " 0.76061776 0.74903475]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['models/subject_mlbb .m']"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.fit(X, y_sub)\n",
    "sub_clf = Pipeline([(\"tfidfVectorizer\",vec),(\"clf\", clf)])\n",
    "# 模型保存\n",
    "joblib.dump(sub_clf, \"models/subject_clf.m\")\n",
    "joblib.dump(subject_mlb, \"models/subject_mlbb .m\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/python3/lib/python3.6/site-packages/sklearn/feature_extraction/text.py:1089: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  if hasattr(X, 'dtype') and np.issubdtype(X.dtype, np.float):\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "价格 [0.85       0.71698113 0.73584906 0.94968553 0.89937107 0.96855346\n",
      " 0.67924528 0.55345912]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/python3/lib/python3.6/site-packages/sklearn/feature_extraction/text.py:1089: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  if hasattr(X, 'dtype') and np.issubdtype(X.dtype, np.float):\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "内饰 [0.58208955 0.46268657 0.50746269 0.70149254 0.73134328 0.58208955\n",
      " 0.40298507 0.41791045]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/python3/lib/python3.6/site-packages/sklearn/feature_extraction/text.py:1089: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  if hasattr(X, 'dtype') and np.issubdtype(X.dtype, np.float):\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "动力 [0.70467836 0.54093567 0.63157895 0.90350877 0.90322581 0.7888563\n",
      " 0.74780059 0.5542522 ]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/python3/lib/python3.6/site-packages/sklearn/feature_extraction/text.py:1089: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  if hasattr(X, 'dtype') and np.issubdtype(X.dtype, np.float):\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "外观 [0.41935484 0.57377049 0.75409836 0.6557377  0.63934426 0.44262295\n",
      " 0.49180328 0.39344262]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/python3/lib/python3.6/site-packages/sklearn/feature_extraction/text.py:1089: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  if hasattr(X, 'dtype') and np.issubdtype(X.dtype, np.float):\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "安全性 [0.76388889 0.55555556 0.84722222 0.77777778 0.75       0.85915493\n",
      " 0.49295775 0.3943662 ]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/python3/lib/python3.6/site-packages/sklearn/feature_extraction/text.py:1089: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  if hasattr(X, 'dtype') and np.issubdtype(X.dtype, np.float):\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "操控 [0.71538462 0.53846154 0.57692308 0.70769231 0.69767442 0.58914729\n",
      " 0.44186047 0.39534884]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/python3/lib/python3.6/site-packages/sklearn/feature_extraction/text.py:1089: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  if hasattr(X, 'dtype') and np.issubdtype(X.dtype, np.float):\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "油耗 [0.81617647 0.65441176 0.7037037  0.88148148 0.84444444 0.81481481\n",
      " 0.56296296 0.61481481]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/python3/lib/python3.6/site-packages/sklearn/feature_extraction/text.py:1089: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  if hasattr(X, 'dtype') and np.issubdtype(X.dtype, np.float):\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "空间 [0.42857143 0.21428571 0.6        0.67272727 0.54545455 0.67272727\n",
      " 0.4        0.38181818]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/python3/lib/python3.6/site-packages/sklearn/feature_extraction/text.py:1089: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  if hasattr(X, 'dtype') and np.issubdtype(X.dtype, np.float):\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "舒适性 [0.57264957 0.56410256 0.66666667 0.81896552 0.75862069 0.70689655\n",
      " 0.43103448 0.44827586]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/python3/lib/python3.6/site-packages/sklearn/feature_extraction/text.py:1089: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  if hasattr(X, 'dtype') and np.issubdtype(X.dtype, np.float):\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "配置 [0.55140187 0.62616822 0.87850467 0.8411215  0.76635514 0.71698113\n",
      " 0.6509434  0.47169811]\n"
     ]
    }
   ],
   "source": [
    "# 每个subject一个情感值分类\n",
    "sent_clfs = {}\n",
    "for subject, gp in df.groupby(\"subject\"):\n",
    "    contents = gp[\"content\"].tolist()\n",
    "    vec_v =TfidfVectorizer(ngram_range=(1, 2), \n",
    "                                        analyzer=\"char\",\n",
    "                                        max_df=0.7, \n",
    "                                        min_df=3)\n",
    "   \n",
    "    X_v = vec_v.fit_transform(contents)    \n",
    "    values =gp[\"sentiment_value\"].tolist()\n",
    "    \n",
    "    clf=RandomForestClassifier(n_estimators=50, max_features=\"auto\")\n",
    "    \n",
    "#     k_fold = KFold(n_splits=8)\n",
    "#     scores = cross_val_score(clf, X_v,values, cv=k_fold, n_jobs=-1)\n",
    "#     print(subject, scores)\n",
    "    \n",
    "    clf.fit(X_v,values)\n",
    "    sent_clfs[subject] =  Pipeline([(\"tfidfVectorizer\",vec_v),(\"clf\", clf)])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>content_id</th>\n",
       "      <th>content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>XuPwKCnA2fqNh5vm</td>\n",
       "      <td>欧蓝德，价格便宜，森林人太贵啦！</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2jNbDn85goX3IuPE</td>\n",
       "      <td>楼主什么时候提的车，南昌优惠多少啊</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>hLgEADQ8sUnvGFK9</td>\n",
       "      <td>吉林，2.5优惠20000，送三年九次保养，贴膜</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>nZmM7LQsfr03wUaz</td>\n",
       "      <td>便宜2万的豪华特装，实用配制提升，优惠还给力，确实划算。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>pwd8MnrthDqLZafe</td>\n",
       "      <td>如果实在想买就等车展期间，优惠2万，我24.98万入的2.5豪</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         content_id                          content\n",
       "0  XuPwKCnA2fqNh5vm             欧蓝德，价格便宜，森林人太贵啦！    \n",
       "1  2jNbDn85goX3IuPE                楼主什么时候提的车，南昌优惠多少啊\n",
       "2  hLgEADQ8sUnvGFK9         吉林，2.5优惠20000，送三年九次保养，贴膜\n",
       "3  nZmM7LQsfr03wUaz     便宜2万的豪华特装，实用配制提升，优惠还给力，确实划算。\n",
       "4  pwd8MnrthDqLZafe  如果实在想买就等车展期间，优惠2万，我24.98万入的2.5豪"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_path = \"test_public.csv\"\n",
    "df_test = pd.read_csv(test_path)\n",
    "df_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_contents = df_test[\"content\"].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/python3/lib/python3.6/site-packages/sklearn/feature_extraction/text.py:1089: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  if hasattr(X, 'dtype') and np.issubdtype(X.dtype, np.float):\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(2364, 21784)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = vec.transform(test_contents)\n",
    "test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/python3/lib/python3.6/site-packages/sklearn/feature_extraction/text.py:1089: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  if hasattr(X, 'dtype') and np.issubdtype(X.dtype, np.float):\n"
     ]
    }
   ],
   "source": [
    "test_subjects_0 = sub_clf.predict(test_contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_subjects = subject_mlb.inverse_transform(test_subjects_0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test[\"subjects\"] = test_subjects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>content_id</th>\n",
       "      <th>content</th>\n",
       "      <th>subjects</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>XuPwKCnA2fqNh5vm</td>\n",
       "      <td>欧蓝德，价格便宜，森林人太贵啦！</td>\n",
       "      <td>(价格,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2jNbDn85goX3IuPE</td>\n",
       "      <td>楼主什么时候提的车，南昌优惠多少啊</td>\n",
       "      <td>(价格,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>hLgEADQ8sUnvGFK9</td>\n",
       "      <td>吉林，2.5优惠20000，送三年九次保养，贴膜</td>\n",
       "      <td>(价格,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>nZmM7LQsfr03wUaz</td>\n",
       "      <td>便宜2万的豪华特装，实用配制提升，优惠还给力，确实划算。</td>\n",
       "      <td>(价格,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>pwd8MnrthDqLZafe</td>\n",
       "      <td>如果实在想买就等车展期间，优惠2万，我24.98万入的2.5豪</td>\n",
       "      <td>(价格,)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         content_id                          content subjects\n",
       "0  XuPwKCnA2fqNh5vm             欧蓝德，价格便宜，森林人太贵啦！        (价格,)\n",
       "1  2jNbDn85goX3IuPE                楼主什么时候提的车，南昌优惠多少啊    (价格,)\n",
       "2  hLgEADQ8sUnvGFK9         吉林，2.5优惠20000，送三年九次保养，贴膜    (价格,)\n",
       "3  nZmM7LQsfr03wUaz     便宜2万的豪华特装，实用配制提升，优惠还给力，确实划算。    (价格,)\n",
       "4  pwd8MnrthDqLZafe  如果实在想买就等车展期间，优惠2万，我24.98万入的2.5豪    (价格,)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "submit = []\n",
    "for i, row in df_test.iterrows():\n",
    "    item = row.to_dict()\n",
    "    del item[\"subjects\"]\n",
    "    content = row[\"content\"]\n",
    "#     if not item:\n",
    "#         print(row)\n",
    "#         continue\n",
    "    for sub in row[\"subjects\"]:\n",
    "        item_new = dict(item)\n",
    "        item_new[\"subject\"] = sub\n",
    "        clf = sent_clfs[sub]\n",
    "        item_new[\"sentiment_value\"] = clf.predict([content])[0]\n",
    "\n",
    "        submit.append(item_new)\n",
    "df_submit = pd.DataFrame(submit)[[\"content_id\", \"content\", \"subject\", \"sentiment_value\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>content_id</th>\n",
       "      <th>content</th>\n",
       "      <th>subject</th>\n",
       "      <th>sentiment_value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>XuPwKCnA2fqNh5vm</td>\n",
       "      <td>欧蓝德，价格便宜，森林人太贵啦！</td>\n",
       "      <td>价格</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2jNbDn85goX3IuPE</td>\n",
       "      <td>楼主什么时候提的车，南昌优惠多少啊</td>\n",
       "      <td>价格</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>hLgEADQ8sUnvGFK9</td>\n",
       "      <td>吉林，2.5优惠20000，送三年九次保养，贴膜</td>\n",
       "      <td>价格</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>nZmM7LQsfr03wUaz</td>\n",
       "      <td>便宜2万的豪华特装，实用配制提升，优惠还给力，确实划算。</td>\n",
       "      <td>价格</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>pwd8MnrthDqLZafe</td>\n",
       "      <td>如果实在想买就等车展期间，优惠2万，我24.98万入的2.5豪</td>\n",
       "      <td>价格</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>cudSKGTyMLbIt8k3</td>\n",
       "      <td>2.0时尚优惠两万现金吗？还有其他赠品吗？</td>\n",
       "      <td>价格</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>U5RMQtopwFzDVxHm</td>\n",
       "      <td>27.5 相比较优惠的少</td>\n",
       "      <td>价格</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Kh0ayJjuTrzwPAs1</td>\n",
       "      <td>综合优惠两万，不是现金优惠两万，送的垃圾东西都包含在内.大忽悠.</td>\n",
       "      <td>价格</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>YZovRtEAyQJgTi1I</td>\n",
       "      <td>差不多15000左右的优惠</td>\n",
       "      <td>价格</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>BeSIAUzoTWH9muYl</td>\n",
       "      <td>恭喜恭喜，这个配置性价比很高</td>\n",
       "      <td>价格</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>BeSIAUzoTWH9muYl</td>\n",
       "      <td>恭喜恭喜，这个配置性价比很高</td>\n",
       "      <td>配置</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>SXYTBWg6FxuboMnd</td>\n",
       "      <td>谁还用车载导航呀！用手机导航吧及时性好！而且到港后装的机头不行，在意的话建议去换一个机头顺便...</td>\n",
       "      <td>配置</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>RDgTmzW9xBfY6pGv</td>\n",
       "      <td>你好，我也是受够了16款的垃圾导航。这个美版主机哪里可以买到？安装后可以完美使用吗，有什么问...</td>\n",
       "      <td>配置</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>8cMwzsxkL4XtuDG1</td>\n",
       "      <td>森林人导航确实垃圾。再好的车再好的导航都没有手机导航好用，我提车后只用了二个多月的车载导航就...</td>\n",
       "      <td>配置</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>MOUrvjDsIJpbw6dQ</td>\n",
       "      <td>导航现在都用手机的，音箱没有说的那么差，就是低音差点，人声其实还可以。</td>\n",
       "      <td>配置</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>t01PVoDSd9Lsy5HN</td>\n",
       "      <td>黑屏经常发生啊和手机蓝牙，连接后电话打进来根本听不到对方在说什么，想指往导航功能就会让你上火</td>\n",
       "      <td>配置</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1rGSYnOx9Fb2ULQX</td>\n",
       "      <td>那你们换的什么中控呀？</td>\n",
       "      <td>配置</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>qIUiOxGdb0M4PQky</td>\n",
       "      <td>接孩子应该就在城区，这油耗正常的。</td>\n",
       "      <td>油耗</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>6GMYQa0Ag4o9nsZ2</td>\n",
       "      <td>平路上定速８０就是５个左右，我新车１０００公里内都是这样跑的，可以的。高速不过１００会更低，...</td>\n",
       "      <td>油耗</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>6Ewe2kxO5msAg0yU</td>\n",
       "      <td>俩车排量都不行，1.5t昂科威动力不行，比2.0t还费油，你图什么？</td>\n",
       "      <td>动力</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          content_id                                            content  \\\n",
       "0   XuPwKCnA2fqNh5vm                               欧蓝德，价格便宜，森林人太贵啦！       \n",
       "1   2jNbDn85goX3IuPE                                  楼主什么时候提的车，南昌优惠多少啊   \n",
       "2   hLgEADQ8sUnvGFK9                           吉林，2.5优惠20000，送三年九次保养，贴膜   \n",
       "3   nZmM7LQsfr03wUaz                       便宜2万的豪华特装，实用配制提升，优惠还给力，确实划算。   \n",
       "4   pwd8MnrthDqLZafe                    如果实在想买就等车展期间，优惠2万，我24.98万入的2.5豪   \n",
       "5   cudSKGTyMLbIt8k3                          2.0时尚优惠两万现金吗？还有其他赠品吗？       \n",
       "6   U5RMQtopwFzDVxHm                                       27.5 相比较优惠的少   \n",
       "7   Kh0ayJjuTrzwPAs1               综合优惠两万，不是现金优惠两万，送的垃圾东西都包含在内.大忽悠.       \n",
       "8   YZovRtEAyQJgTi1I                                  差不多15000左右的优惠       \n",
       "9   BeSIAUzoTWH9muYl                                     恭喜恭喜，这个配置性价比很高   \n",
       "10  BeSIAUzoTWH9muYl                                     恭喜恭喜，这个配置性价比很高   \n",
       "11  SXYTBWg6FxuboMnd  谁还用车载导航呀！用手机导航吧及时性好！而且到港后装的机头不行，在意的话建议去换一个机头顺便...   \n",
       "12  RDgTmzW9xBfY6pGv  你好，我也是受够了16款的垃圾导航。这个美版主机哪里可以买到？安装后可以完美使用吗，有什么问...   \n",
       "13  8cMwzsxkL4XtuDG1  森林人导航确实垃圾。再好的车再好的导航都没有手机导航好用，我提车后只用了二个多月的车载导航就...   \n",
       "14  MOUrvjDsIJpbw6dQ                导航现在都用手机的，音箱没有说的那么差，就是低音差点，人声其实还可以。   \n",
       "15  t01PVoDSd9Lsy5HN     黑屏经常发生啊和手机蓝牙，连接后电话打进来根本听不到对方在说什么，想指往导航功能就会让你上火   \n",
       "16  1rGSYnOx9Fb2ULQX                                    那你们换的什么中控呀？       \n",
       "17  qIUiOxGdb0M4PQky                                  接孩子应该就在城区，这油耗正常的。   \n",
       "18  6GMYQa0Ag4o9nsZ2  平路上定速８０就是５个左右，我新车１０００公里内都是这样跑的，可以的。高速不过１００会更低，...   \n",
       "19  6Ewe2kxO5msAg0yU             俩车排量都不行，1.5t昂科威动力不行，比2.0t还费油，你图什么？       \n",
       "\n",
       "   subject  sentiment_value  \n",
       "0       价格                1  \n",
       "1       价格                0  \n",
       "2       价格                0  \n",
       "3       价格                0  \n",
       "4       价格                0  \n",
       "5       价格                0  \n",
       "6       价格                0  \n",
       "7       价格                0  \n",
       "8       价格                0  \n",
       "9       价格                0  \n",
       "10      配置                0  \n",
       "11      配置                0  \n",
       "12      配置                0  \n",
       "13      配置                0  \n",
       "14      配置                0  \n",
       "15      配置                0  \n",
       "16      配置                0  \n",
       "17      油耗                0  \n",
       "18      油耗                0  \n",
       "19      动力                0  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_submit.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_submit.to_csv(\"submit2.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import jieba\n",
    "# import re\n",
    "# def process_docs(docs, cut_all=True):\n",
    "#     new_docs = []\n",
    "#     for doc in docs:\n",
    "#         new_doc = \" \".join(list(filter(lambda x:bool(re.search(\"\\w\", x)),  jieba.cut(doc,cut_all=cut_all))))\n",
    "#         new_docs.append(new_doc)\n",
    "#     return new_docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#new_train_x = process_docs(train_X_0)\n",
    "\n",
    "# vec=  TfidfVectorizer(token_pattern=r\"(?u)\\b\\S+\\b\",\n",
    "#                                    stop_words=[],\n",
    "#                                    max_df=0.6, \n",
    "#                                    min_df=5)\n",
    "\n",
    "# X = vec.fit_transform(new_train_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# vec = CountVectorizer(binary=True,\n",
    "#                                     token_pattern=r\"(?u)\\b\\S+\\b\",\n",
    "#                                     ngram_range=(1, 4), \n",
    "#                                     analyzer=\"char\",\n",
    "#                                    max_df=0.6, \n",
    "#                                    min_df=5)\n",
    "\n",
    "# # X = vec.fit_transform([re.sub(\"\\W\",\"\", doc) for doc in train_X_0])\n",
    "# X = vec.fit_transform(train_X_0)\n",
    "# #select = SelectKBest(chi2, k=150)\n",
    "# select = SelectPercentile(chi2, 70)\n",
    "# X=select.fit_transform(X, y)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
